{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8a82a769ac88b441",
   "metadata": {},
   "source": [
    "## 数据集下载\n",
    "我们可以利用ModelScope或Huggingface来下载数据集，但是Huggingface（国外）下载比较慢，所以这里我们使用ModelScope（国内，阿里）。\n",
    "\n",
    "数据集地址：\n",
    "https://www.modelscope.cn/datasets/DAMO_NLP/jd/quickstart\n",
    "\n",
    "先安装ModelScope"
   ]
  },
  {
   "cell_type": "code",
   "id": "1b697d47446e075d",
   "metadata": {
    "scrolled": true,
    "ExecuteTime": {
     "end_time": "2025-07-14T15:00:31.815556Z",
     "start_time": "2025-07-14T15:00:31.812238Z"
    }
   },
   "source": [
    "# !pip install \"modelscope[framework]\""
   ],
   "outputs": [],
   "execution_count": 1
  },
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-15T11:36:24.270440Z",
     "start_time": "2025-07-15T11:36:21.201160Z"
    }
   },
   "source": [
    "# 利用modelscope下载数据\n",
    "from modelscope.msdatasets import MsDataset\n",
    "import math\n",
    "\n",
    "ds_train = MsDataset.load('DAMO_NLP/jd', subset_name='default', split='train')\n",
    "\n",
    "# 训练集中有些sentence不是字符串，把它过滤掉\n",
    "ds_train_torch = ds_train.to_torch_dataset().filter(\n",
    "    lambda x: isinstance(x['sentence'], str) and x['label'] is not None and not math.isnan(x['label']))"
   ],
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-07-15 19:36:21,775 - modelscope - WARNING - Use trust_remote_code=True. Will invoke codes from jd. Please make sure that you can trust the external codes.\n",
      "2025-07-15 19:36:23,434 - modelscope - WARNING - Reusing dataset dataset_builder (/Users/dadudu/.cache/modelscope/hub/datasets/DAMO_NLP/jd/master/data_files)\n",
      "2025-07-15 19:36:23,435 - modelscope - INFO - Generating dataset dataset_builder (/Users/dadudu/.cache/modelscope/hub/datasets/DAMO_NLP/jd/master/data_files)\n",
      "2025-07-15 19:36:23,435 - modelscope - INFO - Reusing cached meta-data file: /Users/dadudu/.cache/modelscope/hub/datasets/DAMO_NLP/jd/master/data_files/3a0b7ca43b11a413d66fb247f31fb16f\n",
      "Filter: 100%|██████████| 45366/45366 [00:00<00:00, 94030.73 examples/s] \n"
     ]
    }
   ],
   "execution_count": 2
  },
  {
   "cell_type": "markdown",
   "id": "e5f97589538fef6e",
   "metadata": {},
   "source": [
    "基于ds_train_torch自定义一个MyDataset"
   ]
  },
  {
   "cell_type": "code",
   "id": "775120400f268554",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-15T11:36:59.426399Z",
     "start_time": "2025-07-15T11:36:58.335715Z"
    }
   },
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "from modelscope import AutoTokenizer\n",
    "import torch\n",
    "\n",
    "# 只取部分数据快速训练（全量数据请注释下面两行）\n",
    "ds_train_torch = list(ds_train_torch)[:200]\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"tiansz/bert-base-chinese\")\n",
    "tokenizer.all_special_ids"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading Model from https://www.modelscope.cn to directory: /Users/dadudu/.cache/modelscope/hub/models/tiansz/bert-base-chinese\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[100, 102, 0, 101, 103]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-15T11:37:22.642501Z",
     "start_time": "2025-07-15T11:37:22.639174Z"
    }
   },
   "cell_type": "code",
   "source": "tokenizer.special_tokens_map",
   "id": "8c6ab36f24a04b1c",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'unk_token': '[UNK]',\n",
       " 'sep_token': '[SEP]',\n",
       " 'pad_token': '[PAD]',\n",
       " 'cls_token': '[CLS]',\n",
       " 'mask_token': '[MASK]'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-15T11:39:46.687291Z",
     "start_time": "2025-07-15T11:39:46.682374Z"
    }
   },
   "cell_type": "code",
   "source": [
    "class BertDataset(Dataset):\n",
    "    def __init__(self, dataset):\n",
    "        self.dataset = dataset\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.dataset)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        item = self.dataset[idx]\n",
    "        text = item['sentence']\n",
    "        # 使用BERT分词器编码文本\n",
    "        encoding = tokenizer.encode_plus(\n",
    "            text,\n",
    "            add_special_tokens=True,  # 添加[CLS]和[SEP]\n",
    "            max_length=128,  # 最大长度\n",
    "            padding='max_length',  # 填充到最大长度\n",
    "            truncation=True,  # 截断过长的文本\n",
    "            return_attention_mask=True,  # 返回注意力掩码\n",
    "            return_tensors='pt'  # 返回PyTorch张量\n",
    "        )\n",
    "\n",
    "        return encoding['input_ids'][0], encoding['attention_mask'][0], torch.tensor(item['label'])\n",
    "\n",
    "\n",
    "train_dataset = BertDataset(ds_train_torch)\n",
    "train_loader = DataLoader(train_dataset, batch_size=2, shuffle=False)\n",
    "\n",
    "for input_ids, attention_mask, labels in train_loader:\n",
    "    print(input_ids.shape)\n",
    "    print(attention_mask.shape)\n",
    "    print(labels)\n",
    "    break"
   ],
   "id": "1a6a7a7c7d44f670",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 128])\n",
      "torch.Size([2, 128])\n",
      "tensor([1., 1.])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/vl/mkwcfmqd5kb3rykv5bb3w3n40000gn/T/ipykernel_38510/392864532.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  return encoding['input_ids'][0], encoding['attention_mask'][0], torch.tensor(item['label'])\n"
     ]
    }
   ],
   "execution_count": 12
  },
  {
   "cell_type": "markdown",
   "id": "ab0a855656602b32",
   "metadata": {},
   "source": [
    "## 定义模型"
   ]
  },
  {
   "cell_type": "code",
   "id": "9c8f98d9a8248b3e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-15T11:42:15.426253Z",
     "start_time": "2025-07-15T11:42:15.405750Z"
    }
   },
   "source": [
    "# 定义模型\n",
    "from torch import nn\n",
    "\n",
    "from modelscope import AutoModel\n",
    "\n",
    "class BertForTextClassification(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.bert = AutoModel.from_pretrained(\"tiansz/bert-base-chinese\")\n",
    "        self.fc = nn.Linear(768, 2)\n",
    "\n",
    "    def forward(self, input_ids, attention_mask):\n",
    "        outputs = self.bert(input_ids, attention_mask=attention_mask)\n",
    "        pooled_output = outputs.pooler_output\n",
    "        return self.fc(pooled_output)"
   ],
   "outputs": [],
   "execution_count": 13
  },
  {
   "cell_type": "markdown",
   "id": "fb0b5c0604aa5322",
   "metadata": {},
   "source": [
    "## 开始训练"
   ]
  },
  {
   "cell_type": "code",
   "id": "e61245c07dc2e336",
   "metadata": {
    "scrolled": true,
    "ExecuteTime": {
     "end_time": "2025-07-15T11:45:57.607393Z",
     "start_time": "2025-07-15T11:44:25.001246Z"
    }
   },
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'mps')\n",
    "\n",
    "model = BertForTextClassification()\n",
    "model.to(device)\n",
    "\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=2e-5)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "EPOCHS = 10\n",
    "for epoch in range(EPOCHS):\n",
    "    for i, (input_ids, attention_mask, labels) in enumerate(train_loader):\n",
    "        input_ids = input_ids.to(device)\n",
    "        attention_mask = attention_mask.to(device)\n",
    "        labels = labels.to(device)\n",
    "\n",
    "        outputs = model(input_ids, attention_mask)\n",
    "        loss = criterion(outputs, labels.long())\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        if (i + 1) % 10 == 0:\n",
    "            print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'\n",
    "                  .format(epoch + 1, EPOCHS, i + 1, len(train_loader), loss.item()))"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading Model from https://www.modelscope.cn to directory: /Users/dadudu/.cache/modelscope/hub/models/tiansz/bert-base-chinese\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/vl/mkwcfmqd5kb3rykv5bb3w3n40000gn/T/ipykernel_38510/392864532.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  return encoding['input_ids'][0], encoding['attention_mask'][0], torch.tensor(item['label'])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/10], Step [10/100], Loss: 0.7371\n",
      "Epoch [1/10], Step [20/100], Loss: 0.3044\n",
      "Epoch [1/10], Step [30/100], Loss: 0.5898\n",
      "Epoch [1/10], Step [40/100], Loss: 0.1781\n",
      "Epoch [1/10], Step [50/100], Loss: 0.4124\n",
      "Epoch [1/10], Step [60/100], Loss: 0.1136\n",
      "Epoch [1/10], Step [70/100], Loss: 0.0803\n",
      "Epoch [1/10], Step [80/100], Loss: 0.2624\n",
      "Epoch [1/10], Step [90/100], Loss: 0.3395\n",
      "Epoch [1/10], Step [100/100], Loss: 0.1441\n",
      "Epoch [2/10], Step [10/100], Loss: 0.0893\n",
      "Epoch [2/10], Step [20/100], Loss: 0.0531\n",
      "Epoch [2/10], Step [30/100], Loss: 0.3697\n",
      "Epoch [2/10], Step [40/100], Loss: 0.0183\n",
      "Epoch [2/10], Step [50/100], Loss: 0.0435\n",
      "Epoch [2/10], Step [60/100], Loss: 0.0241\n",
      "Epoch [2/10], Step [70/100], Loss: 0.0164\n",
      "Epoch [2/10], Step [80/100], Loss: 0.0108\n",
      "Epoch [2/10], Step [90/100], Loss: 0.0306\n",
      "Epoch [2/10], Step [100/100], Loss: 0.0209\n",
      "Epoch [3/10], Step [10/100], Loss: 0.0461\n",
      "Epoch [3/10], Step [20/100], Loss: 0.0103\n",
      "Epoch [3/10], Step [30/100], Loss: 0.0199\n",
      "Epoch [3/10], Step [40/100], Loss: 0.0038\n",
      "Epoch [3/10], Step [50/100], Loss: 0.0064\n",
      "Epoch [3/10], Step [60/100], Loss: 0.0028\n",
      "Epoch [3/10], Step [70/100], Loss: 0.0026\n",
      "Epoch [3/10], Step [80/100], Loss: 0.0022\n",
      "Epoch [3/10], Step [90/100], Loss: 0.0034\n",
      "Epoch [3/10], Step [100/100], Loss: 0.0025\n",
      "Epoch [4/10], Step [10/100], Loss: 0.0019\n",
      "Epoch [4/10], Step [20/100], Loss: 0.0025\n",
      "Epoch [4/10], Step [30/100], Loss: 0.0076\n",
      "Epoch [4/10], Step [40/100], Loss: 0.0012\n",
      "Epoch [4/10], Step [50/100], Loss: 0.0039\n",
      "Epoch [4/10], Step [60/100], Loss: 0.0008\n",
      "Epoch [4/10], Step [70/100], Loss: 0.0011\n",
      "Epoch [4/10], Step [80/100], Loss: 0.0007\n",
      "Epoch [4/10], Step [90/100], Loss: 0.0011\n",
      "Epoch [4/10], Step [100/100], Loss: 0.0008\n",
      "Epoch [5/10], Step [10/100], Loss: 0.0007\n",
      "Epoch [5/10], Step [20/100], Loss: 0.0008\n",
      "Epoch [5/10], Step [30/100], Loss: 0.0043\n",
      "Epoch [5/10], Step [40/100], Loss: 0.0005\n",
      "Epoch [5/10], Step [50/100], Loss: 0.0027\n",
      "Epoch [5/10], Step [60/100], Loss: 0.0005\n",
      "Epoch [5/10], Step [70/100], Loss: 0.0006\n",
      "Epoch [5/10], Step [80/100], Loss: 0.0004\n",
      "Epoch [5/10], Step [90/100], Loss: 0.0007\n",
      "Epoch [5/10], Step [100/100], Loss: 0.0005\n",
      "Epoch [6/10], Step [10/100], Loss: 0.0004\n",
      "Epoch [6/10], Step [20/100], Loss: 0.0005\n",
      "Epoch [6/10], Step [30/100], Loss: 0.0027\n",
      "Epoch [6/10], Step [40/100], Loss: 0.0003\n",
      "Epoch [6/10], Step [50/100], Loss: 0.0020\n",
      "Epoch [6/10], Step [60/100], Loss: 0.0003\n",
      "Epoch [6/10], Step [70/100], Loss: 0.0004\n",
      "Epoch [6/10], Step [80/100], Loss: 0.0003\n",
      "Epoch [6/10], Step [90/100], Loss: 0.0005\n",
      "Epoch [6/10], Step [100/100], Loss: 0.0004\n",
      "Epoch [7/10], Step [10/100], Loss: 0.0003\n",
      "Epoch [7/10], Step [20/100], Loss: 0.0003\n",
      "Epoch [7/10], Step [30/100], Loss: 0.0017\n",
      "Epoch [7/10], Step [40/100], Loss: 0.0003\n",
      "Epoch [7/10], Step [50/100], Loss: 0.0016\n",
      "Epoch [7/10], Step [60/100], Loss: 0.0002\n",
      "Epoch [7/10], Step [70/100], Loss: 0.0003\n",
      "Epoch [7/10], Step [80/100], Loss: 0.0002\n",
      "Epoch [7/10], Step [90/100], Loss: 0.0003\n",
      "Epoch [7/10], Step [100/100], Loss: 0.0003\n",
      "Epoch [8/10], Step [10/100], Loss: 0.0002\n",
      "Epoch [8/10], Step [20/100], Loss: 0.0003\n",
      "Epoch [8/10], Step [30/100], Loss: 0.0011\n",
      "Epoch [8/10], Step [40/100], Loss: 0.0002\n",
      "Epoch [8/10], Step [50/100], Loss: 0.0013\n",
      "Epoch [8/10], Step [60/100], Loss: 0.0002\n",
      "Epoch [8/10], Step [70/100], Loss: 0.0002\n",
      "Epoch [8/10], Step [80/100], Loss: 0.0002\n",
      "Epoch [8/10], Step [90/100], Loss: 0.0003\n",
      "Epoch [8/10], Step [100/100], Loss: 0.0002\n",
      "Epoch [9/10], Step [10/100], Loss: 0.0002\n",
      "Epoch [9/10], Step [20/100], Loss: 0.0002\n",
      "Epoch [9/10], Step [30/100], Loss: 0.0008\n",
      "Epoch [9/10], Step [40/100], Loss: 0.0002\n",
      "Epoch [9/10], Step [50/100], Loss: 0.0011\n",
      "Epoch [9/10], Step [60/100], Loss: 0.0002\n",
      "Epoch [9/10], Step [70/100], Loss: 0.0002\n",
      "Epoch [9/10], Step [80/100], Loss: 0.0001\n",
      "Epoch [9/10], Step [90/100], Loss: 0.0002\n",
      "Epoch [9/10], Step [100/100], Loss: 0.0002\n",
      "Epoch [10/10], Step [10/100], Loss: 0.0002\n",
      "Epoch [10/10], Step [20/100], Loss: 0.0002\n",
      "Epoch [10/10], Step [30/100], Loss: 0.0006\n",
      "Epoch [10/10], Step [40/100], Loss: 0.0001\n",
      "Epoch [10/10], Step [50/100], Loss: 0.0009\n",
      "Epoch [10/10], Step [60/100], Loss: 0.0001\n",
      "Epoch [10/10], Step [70/100], Loss: 0.0001\n",
      "Epoch [10/10], Step [80/100], Loss: 0.0001\n",
      "Epoch [10/10], Step [90/100], Loss: 0.0002\n",
      "Epoch [10/10], Step [100/100], Loss: 0.0002\n"
     ]
    }
   ],
   "execution_count": 14
  },
  {
   "cell_type": "code",
   "id": "afd07f2d6a3b3d6f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-15T11:48:27.415280Z",
     "start_time": "2025-07-15T11:48:27.324574Z"
    }
   },
   "source": [
    "model.eval()\n",
    "\n",
    "# test = \"这个手机很好\"\n",
    "# test = \"这个手机很烂\"\n",
    "# test = \"这个手机很不好\"\n",
    "# test = \"这个手机非常非常不好\"\n",
    "test = \"这个手机不好\"\n",
    "encoding = tokenizer.encode_plus(\n",
    "    test,\n",
    "    add_special_tokens=True,\n",
    "    max_length=128,\n",
    "    padding='max_length',\n",
    "    truncation=True,\n",
    "    return_attention_mask=True,\n",
    "    return_tensors='pt'\n",
    ")\n",
    "\n",
    "input_ids = encoding['input_ids']\n",
    "input_ids = input_ids.to(device)\n",
    "attention_mask = encoding['attention_mask']\n",
    "attention_mask = attention_mask.to(device)\n",
    "\n",
    "outputs = model(input_ids, attention_mask)\n",
    "print(outputs)\n",
    "predicted = torch.argmax(torch.softmax(outputs, dim=1), 1)\n",
    "predicted"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 4.2236, -4.2071]], device='mps:0', grad_fn=<LinearBackward0>)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([0], device='mps:0')"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 27
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
